import fix_path

from google.appengine.ext import db, deferred

from google.appengine.api import datastore_errors 
from google.appengine.runtime import DeadlineExceededError

from marketdata.models import *
from marketdata.statistic import TradeHubs, MaxStatsToStore, MaxWorkTime

from datetime import  datetime, timedelta

import logging


def numeric_compare(x, y):
    if x > y:
        return 1
    elif x == y:
        return 0
    else: # x<y
        return -1

bidConversion = {'True' : 'buy', 'False' : 'sell'}


class MailMarketStatReader(object):
    def __init__(self):
        self._mailkeyid = None
        self._delay = 0
        self._currentattach = 0
        self._currentpos = 0
        self._finishtime = None
        self._aborted = False
        pass

    def run(self, mailkeyid, delay = 30):
        self._mailkeyid = mailkeyid
        self._delay = delay
        self._currentattach = 0
        self._currentpos = 0
        self._finishtime = None
        self._aborted = False
        deferred.defer(self._continue, _countdown=self._delay)
        pass

    def _continue(self):
        marketlistIndex = 0
        _attach = 0
        if self._currentpos or self._currentattach:
            marketlistIndex = self._currentpos
            _attach = self._currentattach
        self._finishtime = datetime.today() + MaxWorkTime
        self._aborted = False
        try:
            logging.info("Try read mail with ID = %s." % (self._mailkeyid))
            inboundmail = MarketStatsMail.get_by_key_name(self._mailkeyid)
            _processed = 0
            if inboundmail :
                attachlistLen=len(inboundmail.statsdata)
                if attachlistLen :
                    logging.info("Work with mail from date %s by %s with datasize %d." % (inboundmail.date,inboundmail.sender,inboundmail.datasize))
                    uploaded = datetime.strptime(inboundmail.date.split(' +',1)[0],"%a, %d %b %Y %H:%M:%S")
                    typeID = None
                    solarSystemID = None
                    _toStore = {}
                    if self._currentpos or self._currentattach:
                        logging.info("Found cached info. Starting from attach %s, line in attach %s." % (self._currentattach,self._currentpos))
                        marketlistIndex = self._currentpos
                        _attach = self._currentattach
                    attaches = MarketStatsAttach.get_by_id(inboundmail.statsdata[self._currentattach:], parent = inboundmail)
                    _statkey = ''
                    for attach in attaches:
                        marketlist = attach.statsdata.split("\n")
                        marketlistLen = len(marketlist)
                        logging.info("Work with attach # %d with %d lines. Starting from line %d" % (_attach,marketlistLen,marketlistIndex))
                        while marketlistIndex < marketlistLen :
                            marketstat = marketlist[marketlistIndex].split(',')
                            marketstatlen = len(marketstat)
                            if marketstatlen == 15 or marketstatlen == 14:
                                # for only jita handling
                                if long(marketstat[12]) in TradeHubs:
                                     tdate = datetime.strptime(marketstat[8].split('.',1)[0].split(' ',1)[0],"%Y-%m-%d").date()
                                     if typeID != marketstat[2] or solarSystemID != marketstat[12]:
                                         if typeID != None :
                                             _toStoreItem['sellpriceslen'] = len(_toStoreItem['sellprices'])
                                             _toStoreItem['buypriceslen'] = len(_toStoreItem['buyprices'])
                                             _toStore[_statkey]=_toStoreItem
                                             if len(_toStore) >= MaxStatsToStore :
                                                 try:
                                                     _processed += self.store_result(_toStore)
                                                     if self._aborted :
                                                         deferred.defer(self._continue, _countdown=self._delay)
                                                         logging.info("Prevented aborting at line %d. Processed %d records." % (self._currentpos,_processed))
                                                         return
                                                     self._currentpos=marketlistIndex
                                                     self._currentattach = _attach
                                                     _toStore = {}
                                                 except (DeadlineExceededError, datastore_errors.Timeout):
                                                     deferred.defer(self._continue, _countdown=self._delay)
                                                     logging.info("Aborted at line %d. Processed %d records." % (marketlistIndex,_processed))
                                                     return
                                         typeID = marketstat[2]
                                         solarSystemID = marketstat[12]
                                         _toStoreItem = {}
                                         _toStoreItem['attach'] = _attach
                                         _toStoreItem['position'] = marketlistIndex
                                         _toStoreItem['uploaded'] = uploaded
                                         _toStoreItem['typeID'] = long(typeID)
                                         _toStoreItem['solarSystemID'] = long(solarSystemID)
                                         _toStoreItem['regionID'] = long(marketstat[11])
                                         _toStoreItem['sellprices'] = []
                                         _toStoreItem['buyprices'] = []
                                         _toStoreItem['sellmindate'] = None
                                         _toStoreItem['sellmaxdate'] = None
                                         _toStoreItem['sellvolentered'] = 0
                                         _toStoreItem['sellvolremaining'] = 0
                                         _toStoreItem['buymindate'] = None
                                         _toStoreItem['buymaxdate'] = None
                                         _toStoreItem['buyvolentered'] = 0
                                         _toStoreItem['buyvolremaining'] = 0
                                         _statkey = "%06d%09d" % (_toStoreItem['typeID'],_toStoreItem['solarSystemID'])
                                         bid = None
    #                                                 logging.info("typeID = %s, SS = %s." % (typeID,solarSystemID))

                                     if bid != marketstat[7]:
                                         bid = marketstat[7]
                                         _bid = bidConversion[bid]
                                         _toStoreItem[_bid+'mindate'] = tdate
                                         _toStoreItem[_bid+'maxdate'] = tdate
                                     _toStoreItem[_bid+'prices'].append(long(float(marketstat[0])*100))
                                     if tdate > _toStoreItem[_bid+'maxdate'] :
                                         _toStoreItem[_bid+'maxdate'] = tdate
                                     elif tdate < _toStoreItem[_bid+'mindate']:
                                         _toStoreItem[_bid+'mindate'] = tdate
                                     _toStoreItem[_bid+'volremaining'] += long(float(marketstat[1]))
                                     _toStoreItem[_bid+'volentered'] += long(float(marketstat[5]))            
    #                                             logging.info("Bid = %s, _shift = %d, tdate = %s, maxdate = %s, mindate = %s" % (str(bid),_shift,str(tdate),str(ordersdata[_shift]),str(ordersdata[_shift+1])))
                            marketlistIndex += 1
                        _attach +=1
                        marketlistIndex = 0
                    if len(_toStore) > 0:
                        try:
                            _processed += self.store_result(_toStore)
                            if self._aborted :
                                deferred.defer(self._continue, _countdown=self._delay)
                                logging.info("Prevented aborting at line %d. Processed %d records." % (self._currentpos,_processed))
                                return
                            self._currentpos=0
                            self._currentattach = _attach
                        except (DeadlineExceededError, datastore_errors.Timeout):
                            deferred.defer(self._continue, _countdown=self._delay)
                            logging.info("Aborted at line %d. Processed %d records." % (marketlistIndex,_processed))
                            return 
                    db.run_in_transaction(self._deletemail,inboundmail)
        except (DeadlineExceededError, datastore_errors.Timeout):
            self._currentpos = marketlistIndex
            self._currentattach = _attach
            deferred.defer(self._continue, _countdown=self._delay)
            logging.info("Aborted. Processed %d records." % (_processed))
            return 
        else:
            logging.info("Finished. Processed %d records." % (_processed))
        pass

    def _deletemail(self,_inboundmail):
        _attaches = MarketStatsAttach.get_by_id(_inboundmail.statsdata, parent = _inboundmail)
        _attaches.insert(0,_inboundmail.key())
        db.delete(_attaches)
        pass

    def store_result(self,_dataToStore):
        _storeData = []
        _storedKeys = []
        for statkey,_dataToStoreItem  in _dataToStore.iteritems():
            logging.info("Try to store: typeID = %d, solarSystemID = %d, statkey = %s" % (_dataToStoreItem['typeID'],_dataToStoreItem['solarSystemID'],statkey))
            if datetime.today() > self._finishtime :
                self._aborted = True
                self._currentattach = _dataToStoreItem['attach']
                self._currentpos = _dataToStoreItem['position']
                break
            _date = _dataToStoreItem['uploaded'].date()
            historykey = statkey + _date.isoformat()
            marketstat = MarketStats.get_by_key_name(statkey)
            _put = True
    #        logging.info("marketstat = %s",(str(marketstat)))
            if marketstat == None:
               _statFound = False
    #           logging.info("New entry in MarketStats: typeID = %d, solarSystemID = %d" % (_typeID,_solarSystemID))
               marketstat = MarketStats(typeID = _dataToStoreItem['typeID'], 
                                      solarSystemID = _dataToStoreItem['solarSystemID'],
                                      key_name = statkey)
               marketstat.regionID = _dataToStoreItem['regionID']
               markethistory = None
            else: 
               _statFound = True
    #           logging.info("Updated entry in MarketStats: typeID = %d, solarSystemID = %d" % (_typeID,_solarSystemID))
               if (marketstat.updated.date() > _date)  or ((marketstat.ordersSell == _dataToStoreItem['sellpriceslen']) and 
                                                            (marketstat.volSellRemaining == _dataToStoreItem['sellvolremaining']) and 
                                                            (marketstat.volSellEntered == _dataToStoreItem['sellvolentered'] ) and 
                                                            (marketstat.newestSellOrder == _dataToStoreItem['sellmindate'] ) and
                                                            (marketstat.ordersSell == _dataToStoreItem['buypriceslen']) and 
                                                            (marketstat.volSellRemaining == _dataToStoreItem['buyvolremaining']) and 
                                                            (marketstat.volSellEntered == _dataToStoreItem['buyvolentered'] ) and 
                                                            (marketstat.newestSellOrder == _dataToStoreItem['buymindate'] )
                                                            ):
    #               logging.info("The same data. No need update!")
                   _put = False
               else:
                   markethistory = MarketHistory.get_by_key_name(historykey)
            if _put:
                marketstat.updated = _dataToStoreItem['uploaded']

                marketstat.ordersSell, marketstat.minSellPrice, marketstat.maxSellPrice, marketstat.averageSellPrice, marketstat.medianSellPrice, marketstat.volSellEntered, marketstat.volSellRemaining, marketstat.oldestSellOrder, marketstat.newestSellOrder = self.ordersstat(_dataToStoreItem,'sell',_dataToStoreItem['sellpriceslen'])
                marketstat.ordersBuy, marketstat.minBuyPrice, marketstat.maxBuyPrice, marketstat.averageBuyPrice, marketstat.medianBuyPrice, marketstat.volBuyEntered, marketstat.volBuyRemaining, marketstat.oldestBuyOrder, marketstat.newestBuyOrder = self.ordersstat(_dataToStoreItem,'buy',_dataToStoreItem['buypriceslen'])
        
                if markethistory == None:
    #               logging.info("New entry in MarketHistory: date = %s, typeID = %d, solarSystemID = %d" % (_date.isoformat(),_typeID,_solarSystemID))
                   markethistory = MarketHistory(typeID = marketstat.typeID, 
                                                solarSystemID = marketstat.solarSystemID,
                                                date = _date, 
                                                key_name = historykey)
                   markethistory.regionID = marketstat.regionID
    #            else:
    #               logging.info("Updated entry in MarketHistory: date = %s, typeID = %d, solarSystemID = %d" % (_date.isoformat(),_typeID,_solarSystemID))
                markethistory.updated = marketstat.updated
                markethistory.ordersSell = marketstat.ordersSell
                markethistory.minSellPrice = marketstat.minSellPrice
                markethistory.maxSellPrice = marketstat.maxSellPrice
                markethistory.averageSellPrice = marketstat.averageSellPrice
                markethistory.medianSellPrice = marketstat.medianSellPrice
                markethistory.volSellRemaining = marketstat.volSellRemaining
                markethistory.volSellEntered = marketstat.volSellEntered
                markethistory.newestSellOrder = marketstat.newestSellOrder
                markethistory.oldestSellOrder = marketstat.oldestSellOrder
                markethistory.ordersBuy = marketstat.ordersBuy
                markethistory.minBuyPrice = marketstat.minBuyPrice
                markethistory.maxBuyPrice = marketstat.maxBuyPrice
                markethistory.averageBuyPrice = marketstat.averageBuyPrice
                markethistory.medianBuyPrice = marketstat.medianBuyPrice
                markethistory.volBuyRemaining = marketstat.volBuyRemaining
                markethistory.volBuyEntered = marketstat.volBuyEntered
                markethistory.newestBuyOrder = marketstat.newestBuyOrder
                markethistory.oldestBuyOrder = marketstat.oldestBuyOrder
                _storeData.append(marketstat)
                _storeData.append(markethistory)
                _storedKeys.append(statkey)
        _storeAmount = len(_storeData)
        if _storeAmount > 0:
            db.put(_storeData)
            logging.info("Stored %d records." % (_storeAmount))
        return(_storeAmount)

    def ordersstat(self,_itemToStore,_bid,_len):
        if _len > 0:
            _itemToStore[_bid+'prices'].sort(cmp=numeric_compare)
            minPrice = _itemToStore[_bid+'prices'][0]
            maxPrice = _itemToStore[_bid+'prices'][-1]
            averagePrice = sum(_itemToStore[_bid+'prices']) / _len
            if not _len % 2 :
                medianPrice = (_itemToStore[_bid+'prices'][(_len/2)-1] + _itemToStore[_bid+'prices'][_len/2]) / 2
            else:
                medianPrice = _itemToStore[_bid+'prices'][_len/2]
            volEntered = _itemToStore[_bid+'volentered']
            volRemaining = _itemToStore[_bid+'volremaining']
            oldestOrder= _itemToStore[_bid+'mindate']
            newestOrder = _itemToStore[_bid+'maxdate']
            return(_len, minPrice, maxPrice, averagePrice, medianPrice, volEntered, volRemaining, oldestOrder, newestOrder)
        else:
            return(0,0,0,0,0,0,0,None,None)